package com.hyj.spark.CF

import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, Dataset, RelationalGroupedDataset, SparkSession}
import org.apache.spark.sql.functions._

import scala.collection.mutable

/**
  *
  * item ij相似度itemCos ij=  同时喜欢物品1 2 的用户数/（喜欢物品1的用户数+喜欢物品2的用户数）
  * item j打分=itemCos*用户对i的rating
  * 注意：如果同一候选物品j和用户多个物品都相似则得分叠加
  */
object ItemBaseHyj {

  def main(args: Array[String]): Unit = {
    val spark: SparkSession = SparkSession.builder().master("local[*]").appName("itemCF")
      .enableHiveSupport().getOrCreate()

    /**
      * 1 计算物品相似度
      */
    val df: DataFrame = spark.sql("select user_id,item_id, rating from badou.udata")
    val df_j: DataFrame = df.selectExpr("user_id", "item_id as item_id_j", "rating as rating_j")
    //获取item i j
    val item_ij_df = df.join(df_j, "user_id")
      //过滤掉i==j的item
      .filter(row=>{
        !row.getAs[String]("item_id").equals(row.getAs[String]("item_id_j"))
      })
      .groupBy("item_id", "item_id_j").count().selectExpr("item_id", "item_id_j")

    //计算喜欢物品i的user_id和人数
    //      val item_count_df: DataFrame = df.groupBy("item_id").count()
    import spark.implicits._
    //    val item_i_df: DataFrame = df.rdd.map(row => {
    //      (row.getAs[String]("item_id"),
    //        (row.getAs[String]("user_id"), row.getAs[String]("rating"))
    //      )
    //    })
    //      .groupByKey().mapValues(v => {
    //      (v.toArray, v.toArray.length)
    //    }).toDF("item_id", "arr_count")

    val item_i_df: DataFrame = df.groupBy("item_id").agg(
      collect_list(col("user_id")).as("user_ids"),
      count("user_id").as("count")
    )


    val item_j_df: DataFrame = item_i_df
      .selectExpr("item_id as item_id_j", "user_ids as user_ids_j", "count as count_j")
    //item i j 关联i和j对应的user集合 item_id item_id_j arr_count arr_count_j
    val itemij_userarr_df: DataFrame = item_ij_df
      .join(item_i_df, "item_id").join(item_j_df, "item_id_j")

    //    itemij_userarr_df.show(false)
    /**
      * +---------+-------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+
      * |item_id_j|item_id|user_ids                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |count|user_ids_j                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |count_j|
      * +---------+-------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+
      * |332      |302    |[191, 49, 54, 62, 206, 197, 214, 190, 303, 181, 154, 31, 305, 178, 102, 100, 144, 13, 315, 88, 282, 334, 208, 112, 277, 145, 392, 276, 360, 126, 319, 47, 146, 378, 131, 171, 201, 192, 345, 204, 107, 222, 433, 229, 339, 427, 29, 420, 43, 268, 241, 397, 383, 346, 461, 418, 405, 179, 362, 10, 90, 297, 425, 413, 149, 430, 232, 104, 285, 129, 26, 514, 15, 293, 489, 421, 343, 498, 460, 390, 173, 140, 269, 322, 567, 466, 446, 404, 408, 299, 116, 532, 507, 423, 569, 352, 512, 634, 3, 375, 561, 450, 63, 628, 570, 105, 555, 548, 6, 574, 509, 414, 94, 139, 531, 515, 367, 14, 685, 111, 707, 697, 666, 331, 543, 384, 683, 327, 721, 431, 520, 762, 16, 490, 464, 134, 640, 619, 52, 717, 74, 69, 271, 364, 177, 704, 401, 474, 207, 655, 758, 818, 284, 473, 669, 720, 788, 216, 747, 853, 755, 151, 328, 828, 445, 329, 838, 664, 894, 388, 733, 808, 871, 889, 370, 399, 389, 463, 751, 448, 833, 158, 592, 787, 676, 713, 775, 865, 696, 428, 809, 870, 458, 896, 866, 743, 422, 913, 416, 740, 210, 744, 802, 451, 615, 605, 535, 547, 827, 782, 752, 526, 363, 656, 71, 845, 920, 249, 657, 819, 905, 480, 854, 524, 934, 240, 537, 801, 485, 902, 624, 626, 860, 556, 454, 724, 851, 560, 48, 654, 668, 564, 115, 278, 791, 688, 587, 883, 863, 770, 880, 806, 160, 846, 926, 617, 940, 695, 745, 877, 784, 40, 651, 841, 842, 393, 39, 380, 836, 781, 875, 820, 616, 915, 772, 2, 185, 903, 332, 611, 467, 544, 486, 812, 551, 919, 475, 931, 710, 147, 673, 874, 635, 344, 417, 898, 186]                                                                                                                                                                                                                                                                                                                                    |297  |[140, 29, 224, 193, 201, 130, 281, 203, 126, 3, 110, 191, 174, 241, 332, 116, 323, 328, 165, 324, 262, 284, 345, 34, 446, 428, 416, 258, 35, 223, 294, 462, 397, 119, 451, 382, 102, 404, 372, 186, 426, 525, 109, 544, 155, 531, 353, 505, 509, 13, 99, 608, 351, 637, 619, 629, 319, 551, 11, 663, 519, 413, 459, 427, 314, 526, 626, 220, 547, 688, 197, 28, 730, 399, 400, 574, 759, 486, 587, 104, 735, 655, 173, 683, 362, 592, 489, 772, 768, 181, 872, 450, 654, 206, 646, 894, 863, 826, 910, 529, 46, 920, 853, 464, 870, 732, 925, 628, 721, 816, 255, 808, 540, 393, 875, 740, 112, 762, 431, 751, 276, 919, 801, 678, 784, 752, 851, 827, 860, 812, 791, 682, 758, 651, 724, 631, 589, 907, 515, 532, 615, 782, 178]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |143    |
      * |288      |346    |[197, 94, 293, 71, 125, 99, 130, 287, 145, 235, 294, 156, 363, 284, 179, 146, 269, 49, 104, 214, 288, 116, 299, 334, 133, 126, 392, 458, 511, 416, 474, 526, 546, 414, 285, 241, 385, 561, 425, 624, 445, 142, 542, 631, 221, 456, 646, 353, 171, 206, 665, 149, 397, 201, 418, 702, 276, 141, 466, 532, 655, 13, 657, 656, 369, 319, 724, 54, 783, 782, 695, 537, 619, 551, 595, 611, 389, 548, 616, 112, 871, 482, 322, 640, 592, 752, 894, 574, 880, 758, 489, 485, 863, 845, 851, 883, 819, 710, 818, 920, 557, 834, 578, 805, 874, 40, 915, 833, 544, 100, 913, 936, 903, 682, 784, 683, 768, 339, 519, 462, 729, 3, 808, 828, 166, 271]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |126  |[286, 37, 62, 282, 165, 258, 293, 176, 74, 274, 294, 197, 163, 191, 285, 228, 178, 100, 16, 166, 339, 239, 220, 21, 206, 79, 342, 63, 193, 240, 46, 363, 348, 327, 332, 304, 117, 303, 257, 68, 347, 344, 174, 270, 292, 141, 186, 377, 168, 127, 368, 76, 222, 142, 314, 378, 387, 409, 255, 4, 283, 276, 99, 126, 73, 104, 214, 66, 364, 92, 7, 130, 362, 272, 367, 94, 86, 341, 181, 329, 251, 280, 349, 129, 335, 234, 268, 291, 170, 50, 299, 397, 281, 324, 105, 416, 394, 118, 351, 201, 447, 425, 412, 227, 250, 396, 32, 391, 408, 135, 467, 345, 203, 430, 446, 264, 450, 375, 154, 365, 436, 259, 19, 241, 458, 509, 453, 3, 260, 533, 504, 320, 420, 472, 346, 515, 478, 215, 521, 144, 439, 487, 479, 256, 400, 441, 262, 546, 150, 403, 36, 246, 54, 470, 485, 116, 392, 476, 592, 589, 542, 297, 336, 455, 560, 334, 261, 278, 110, 422, 59, 437, 463, 229, 451, 308, 177, 632, 418, 519, 82, 626, 510, 579, 317, 14, 399, 497, 417, 648, 633, 107, 300, 495, 556, 549, 606, 588, 582, 596, 155, 462, 501, 47, 323, 614, 498, 179, 555, 442, 587, 45, 620, 619, 78, 557, 570, 531, 388, 673, 372, 610, 688, 643, 511, 679, 550, 28, 64, 609, 167, 101, 57, 525, 624, 537, 188, 424, 204, 343, 113, 743, 654, 617, 24, 139, 489, 674, 376, 429, 566, 666, 733, 655, 474, 758, 605, 279, 628, 630, 783, 81, 676, 681, 493, 526, 781, 702, 724, 782, 355, 752, 102, 812, 529, 695, 740, 650, 552, 109, 551, 768, 761, 830, 405, 847, 721, 715, 663, 841, 770, 601, 687, 862, 741, 496, 843, 466, 706, 757, 69, 627, 791, 457, 652, 482, 849, 772, 119, 826, 27, 518, 790, 793, 697, 171, 750, 599, 634, 729, 898, 894, 836, 434, 502, 449, 38, 711, 908, 374, 915, 593, 903, 801, 544, 616, 872, 820, 910, 13, 834, 595, 921, 366, 831, 414, 863, 618, 760, 886, 854, 578, 755, 816, 265, 810, 853, 432, 393, 923, 805, 143, 611, 926, 586, 699, 900, 880, 717, 507, 930, 927, 693, 574, 223, 785, 486, 668, 808, 123, 824, 629, 2, 200, 835, 703, 445, 642, 464, 735, 569, 887, 580, 218, 732, 773, 428, 671, 747, 877, 802, 827, 870, 806, 460, 39, 631, 301, 608, 904, 677, 846, 725, 928, 734, 534, 683, 435, 160, 876, 452, 922, 875, 924, 814, 709, 832, 488, 859, 33, 920, 213, 395, 448, 828, 938, 645, 817, 893, 925, 356, 777, 897, 682, 221, 789, 864, 739, 882, 825, 818, 554, 787, 34, 603, 646, 842, 26, 919, 896, 892, 315, 407, 604, 869, 52, 804, 685, 623, 548, 833, 913, 404, 140, 190, 159, 907, 20]|478    |
      * |537      |1014   |[130, 195, 251, 174, 269, 318, 250, 303, 292, 270, 198, 223, 38, 109, 152, 392, 345, 92, 387, 297, 117, 417, 474, 342, 374, 429, 486, 334, 94, 26, 184, 500, 219, 506, 393, 463, 314, 514, 416, 344, 468, 262, 653, 552, 141, 548, 560, 569, 521, 540, 525, 592, 711, 363, 599, 642, 543, 580, 726, 159, 768, 854, 523, 761, 294, 385, 859, 582, 472, 445, 647, 655, 435, 917, 590, 501, 936, 889, 938, 805, 714, 593, 793, 927, 654, 895, 919, 782, 851, 880, 459, 757, 886, 869, 916, 790, 870, 286]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |98   |[201, 344, 94, 7, 436, 269, 244, 363, 405, 119, 561, 312, 504, 669, 429, 655, 334, 774, 606, 805, 222, 854, 327, 417, 751, 6, 308, 286, 401, 916]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |30     |
      * |
      */
    // 对2个集合求交集/2个集合的元素总数=相似度sim
    val item_sim_df = itemij_userarr_df.map(row => {
      val user_ids = row.getAs[mutable.WrappedArray[String]]("user_ids").toSet
      val user_ids_j = row.getAs[mutable.WrappedArray[String]]("user_ids_j").toSet
      val sim_count = (user_ids & user_ids_j).size.toDouble
      val sum_count=(user_ids|user_ids_j).size

      (row.getAs[String]("item_id"), row.getAs[String]("item_id_j"), sim_count / sum_count)
    }).selectExpr("_1 as item_id", "_2 as item_id_j", "_3 as sim")
    //    item_sim_df.show(false)
    /**
      * // 查询user_id=385的item_id集合 等值关联(item_id)上一步的结果
      * // score=rating*sim group by item_id_j sum(score)
      * //    排序 top-N
      */
      //i j sim rating
      val item_set: mutable.Set[String] = mutable.Set[String]()
      val set_braod: Broadcast[mutable.Set[String]] = spark.sparkContext.broadcast(item_set)
      df.where("user_id=385").foreach(row=>{
        set_braod.value.add(row.getAs[String]("item_id"))
        println()
      })

    val rec_item_df: DataFrame = df.where("user_id=385").join(item_sim_df, "item_id")
      //过滤用户已有的物品
      .selectExpr("*", "cast(rating as double)*sim as score")
      .groupBy("item_id_j")
      .agg(
        sum("score").as("score_sum")
      )
      .filter(row=>{
        val j: String = row.getAs[String]("item_id_j")
        !set_braod.value.contains(j)
      })
      .selectExpr("item_id_j","score_sum")

      .orderBy(col("score_sum").desc)

    rec_item_df.show(false)
    /**
      * +---------+------------------+
      * |item_id_j|score_sum         |
      * +---------+------------------+
      * |179      |143.17914198698193|
      * |69       |140.99195852299394|
      * |196      |140.07374447863072|
      * |655      |138.07802917674098|
      * |202      |137.9064000660943 |
      * |64       |137.35048128335558|
      * |203      |137.2450135561807 |
      * |96       |136.9428889497994 |
      * |97       |136.65276603864658|
      * |28       |135.93185002890786|
      * |70       |135.19236796694054|
      * |228      |134.3967945982552 |
      * |527      |133.22976794227267|
      * |188      |132.75564372770424|
      * |22       |132.40901034141743|
      * |161      |131.09361040039147|
      * |154      |129.8003258933549 |
      * |265      |129.68535699309507|
      * |11       |129.50250090281733|
      * |193      |129.45160791735313|
      * +---------+------------------+
      */
  }

}
